package com.lmpop.frame.util;

import com.redfin.sitemapgenerator.ChangeFreq;
import com.redfin.sitemapgenerator.SitemapGeneratorBuilder;
import com.redfin.sitemapgenerator.W3CDateFormat;
import com.redfin.sitemapgenerator.WebSitemapUrl;
import com.redfin.sitemapgenerator.W3CDateFormat.Pattern;
import com.redfin.sitemapgenerator.WebSitemapGenerator;
import com.redfin.sitemapgenerator.WebSitemapUrl.Options;
import java.io.File;
import java.io.PrintStream;
import java.net.MalformedURLException;
import java.util.Date;
import java.util.LinkedList;
import java.util.Queue;
import java.util.TimeZone;
import org.apache.commons.collections.CollectionUtils;
import org.htmlparser.Parser;
import org.htmlparser.filters.NodeClassFilter;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeIterator;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;

public class HtmlCrawler
{
  private NodeClassFilter LINK_FILTER = new NodeClassFilter(LinkTag.class);

  private Parser parser = null;
  private Config cfg = new Config();
  private File dir = null;
  private final String BASE_PREFIX = "http://www.lmpop.com/";
  private WebSitemapGenerator sitemapGenerator = null;

  public HtmlCrawler()
  {
    this.dir = new File(this.cfg.getString("rootpath"));

    W3CDateFormat dateFormat = new W3CDateFormat(W3CDateFormat.Pattern.DAY);
    dateFormat.setTimeZone(TimeZone.getTimeZone("GMT+8"));
    try
    {
      this.sitemapGenerator = ((WebSitemapGenerator)((SitemapGeneratorBuilder)((SitemapGeneratorBuilder)WebSitemapGenerator.builder("http://www.lmpop.com/", this.dir).dateFormat(dateFormat)).maxUrls(200)).build());
    }
    catch (MalformedURLException e) {
      System.out.println("the start url [http://www.lmpop.com/] is malformed");
    }
  }

  public boolean check(String url, String basePrefix)
  {
    boolean result = (url != null) && (!url.equals("")) ? url.startsWith(basePrefix) : false;
    return result;
  }

  public void crawl(Queue<ExtWebSiteMapUrl> queue, WebSitemapGenerator sitemapGenerator)
  {
    if (CollectionUtils.isEmpty(queue)) {
      return;
    }

    Queue crawled = new LinkedList();

    this.parser = new Parser();
    do
    {
      ExtWebSiteMapUrl url = (ExtWebSiteMapUrl)queue.poll();

      crawled.add(url);
      if ((url == null) || (!url.canCrawl())) continue;
      try {
        this.parser.setURL(url.getUrlStr());
        NodeList list = this.parser.parse(this.LINK_FILTER);

        NodeIterator iterator = list.elements();
        while (iterator.hasMoreNodes()) {
          LinkTag node = (LinkTag)iterator.nextNode();
          String link = node.getLink();

          ExtWebSiteMapUrl newUrl = null;
          try
          {
            newUrl = new ExtWebSiteMapUrl((WebSitemapUrl.Options)((WebSitemapUrl.Options)new WebSitemapUrl.Options(link).lastMod(new Date())).changeFreq(ChangeFreq.DAILY));
          }
          catch (MalformedURLException e)
          {
            System.out.println("the url [" + link + "] is malformed");
          }

          if ((check(link, "http://www.lmpop.com/")) && (!queue.contains(newUrl)) && (!crawled.contains(newUrl)))
          {
            System.out.println("Log: newUrl = " + newUrl.getUrlStr());
            queue.add(newUrl);
            sitemapGenerator.addUrl(newUrl);
          }
        }
      }
      catch (ParserException e)
      {
        System.out.println("can not parse the url :" + url.getUrl());
      } finally {
        url.disble();
      }
    }

    while (queue.size() > 0);

    sitemapGenerator.write();
    sitemapGenerator.writeSitemapsWithIndex();
  }

  public boolean generator()
  {
    try
    {
      String url = "http://www.lmpop.com";
      ExtWebSiteMapUrl startUrl = new ExtWebSiteMapUrl((WebSitemapUrl.Options)((WebSitemapUrl.Options)new WebSitemapUrl.Options(url).lastMod(new Date())).changeFreq(ChangeFreq.DAILY));

      Queue queue = new LinkedList();
      queue.add(startUrl);

      crawl(queue, this.sitemapGenerator);

      return true;
    }
    catch (MalformedURLException e) {
      System.out.println("the startUrl is malformed");
    }return false;
  }
}